﻿using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Data;
using System.Data.SqlClient;
using System.Diagnostics;
using System.Diagnostics.Eventing.Reader;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Dapper;
using DotnetSpider.DataFlow;
using DotnetSpider.DataFlow.Parser;
using DotnetSpider.DataFlow.Storage;
using DotnetSpider.Http;
using DotnetSpider.Lawyee.Src;
using DotnetSpider.RequestSupplier;
using DotnetSpider.Scheduler.Component;
using DotnetSpider.Selector;
using DotnetSpider.Statistics.Store;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;

using MySql.Data.MySqlClient;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RabbitMQ.Client;
using Serilog;
using StringContent = DotnetSpider.Http.StringContent;

namespace DotnetSpider.Sample.samples
{
	public class FaLvFaGuichinalaw : Spider
	{
		public static async Task RunAsync()
		{
			//var builder = Builder.CreateDefaultBuilder<LawSpider>();  
			var builder = Builder.CreateDefaultBuilder<FaLvFaGuichinalaw>(options =>
			{
				// 每秒 1 个请求
				options.Speed = 10;

				////最大深度
				//options.Depth = 1;

				//最大的队列并发数
				options.RequestedQueueCount = 1000 * 10 * 10 * 10 * 1000;

				// 请求超时
				options.RequestTimeout = 10 * 24 * 60 * 60;

				//尝试次数
				options.RetriedTimes = 50;

				//Scheduler持续时间
				options.EmptySleepTime = 60 * 60 * 24 * 15;



				//爬虫是否使用代理
				//options.UseProxy = true;

				//去除外链
				//options.RemoveOutboundLinks = true; 

			});


			builder.UseSerilog();
			builder.UseQueueDistinctBfsScheduler<HashSetDuplicateRemover>();
			await builder.Build().RunAsync();
		}
		public static SpiderOptions Options_;
		public FaLvFaGuichinalaw(IOptions<SpiderOptions> options,
			SpiderServices services,
			ILogger<Spider> logger, IStatisticsStore _statisticsStore) : base(
			options, services, logger, _statisticsStore)
		{
			Options_ = options.Value;
		}

		protected override async Task InitializeAsync(CancellationToken stoppingToken)
		{

			//string a = "  df 第电放条酷酷酷第烦烦烦条aaa";
			//string content = a.Replace(" ", "");
			//Regex reg = new Regex("第.*?条");
			//MatchCollection mc = Regex.Matches(content, "第.*?条");
			//string[] strArray = new string[mc.Count];
			//for (int i = 0; i < mc.Count; i++)
			//{
			//	strArray[i] = mc[i].Groups[0].Value;
			//}
			//string[] c = content.Split(strArray, StringSplitOptions.RemoveEmptyEntries);


			

			//string a1 = a + " asd .df ff" +b + "&nbsp;" + b + " asd df ff"+ a + "&#32;" + " asd df 。ff" + d + " asd df ff" + d;
			//string a = "	";
			//string b = "	";
			//string c = "　";
			//string d = " ";
			//string b1 = a1.Replace(a,"    ").Replace(b, "    ").Replace(c, "    ").Replace("\\n", "    ").Replace("\\r", "    ");
			//MatchCollection mc = Regex.Matches(a1, "&.*?;");
			//for (int i = 0; i < mc.Count; i++)
			//{
			//	b1 = b1.Replace(mc[i].Groups[0].Value, "    ");
			//}
			//string[] c1 = b1.Split('。');


			//new Regex("[\\s]+").Replace(a1, ",");



			AddDataFlow(new ListNewsParser());
			AddDataFlow(new NewsParser());
			//AddDataFlow(new MyConsoleStorage());

			AddDataFlow(new SqlServerStorage());   //SQL Server存储类


			

			StartAuto();
		}

		public void StartAuto()
		{
			try
			{
				StartLFZZ();

			}
			catch (Exception ex)
			{

				throw ex;
			}

		}

		public async Task<int> StartLFZZ()
		{
			List<Request> listRequest = new List<Request>();
			string Listpage = "1";
			DateTime start = Convert.ToDateTime("1975-01-01");
			DateTime end = Convert.ToDateTime("2020-07-15");
			TimeSpan sp = end.Subtract(start);
			
			for (int i = 0; i <= sp.Days; i++)
			{
				string start2 = start.AddDays(i).ToString("yyyy-MM-dd");
				string end2 = start.AddDays(i + 1).ToString("yyyy-MM-dd");
				Request req = new Request($"http://search.chinalaw.gov.cn/AdvanceSearchResult?SiteID=124&PageIndex={Listpage}&c1={start2}&c2={end2}&c3=&c4=&title=&Query=");
				req.SetProperty("start", start2);
				req.SetProperty("end", end2);
				req.SetProperty("Listpage", Listpage);
				req.Name = "list";
				listRequest.Add(req);
			}

			return await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray());
		}

		public class SqlServerStorage : StorageBase
		{
			public override async Task InitAsync()
			{
			}


			protected override async Task StoreAsync(DataContext context)
			{
				string Detailpage = context.Request.GetProperty("Detailpage");
				var typeName = typeof(News).FullName;
				News data = (News)context.GetData(typeName);
				try
				{
					if (data != null)
					{
						using (var conn = new SqlConnection(Options_.StorageConnectionString))
						{
							
							var have = conn.ExecuteScalar($" select count(1) from Law where  LawID =@LawID ", new { LawID = data.LawID });
							if (have.ToString() == "0")
							{
								if (data.Detailpage == "1")
								{
									conn.Execute($"INSERT INTO Law ([LawID], [Title],[Organ],[StartDate],[ImplementDate],[Type],[Valid],[Other],[Path],[LawType]) VALUES (@LawID, @Title,@Organ,@StartDate,@ImplementDate,@Type,@Valid,@Other,@Path,@LawType) ", data);
								}
								foreach (var item in data.newscontents)
								{
									var have1 = conn.ExecuteScalar($" select count(1) from LawDetail where  Orders =@Orders ", new { Orders = item.Orders });
									if (have1.ToString() == "0")
									{
										conn.Execute($"INSERT INTO LawDetail ([ContentId], [LawID],[PathName],[Content],[Orders]) VALUES (@ContentId, @LawID,@PathName,@Content,@Orders) ", item);
									}

								}
							}
							
						}
					}
				}
				catch (Exception ex)
				{
					using (var conn = new SqlConnection(Options_.StorageConnectionString))
					{
						conn.Execute($"delete from LawDetail  WHERE LawID = '" + data.LawID + "' ");
						conn.Execute($"delete from Law WHERE LawID = '" + data.LawID + "'");
						if (data != null)
						{
							conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message+ex.StackTrace) + ",data:" + CommonHelper.NoHtml(data.LawID + data.Title)+ "','" + DateTime.Now.ToString() + "') ");
						}
						else
						{
							conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message+ex.StackTrace) + ",  :SqlServerStorage,StoreAsync','" + DateTime.Now.ToString() + "') ");
						}
					}
					throw;
				}

			}





		}

		protected override (string Id, string Name) GetIdAndName()
		{
			return (Guid.NewGuid().ToString(), "falvfagui");
		}

 

		protected class ListNewsParser : DataParser
		{
			public ListNewsParser()
			{
				//AddRequiredValidator("search\\.chinalaw\\.gov\\.cn/AdvanceSearchResult");
				AddRequiredValidatorEx("list");

			}


			protected override Task Parse(DataContext context)
			{
				try
				{
					if (context.Selectable.SelectList(Selectors.XPath(".//div[@class='w_list']")) != null)
					{
						var newsList = context.Selectable.SelectList(Selectors.XPath(".//div[@class='w_list']"));
						foreach (var news in newsList)
						{
							//http://search.chinalaw.gov.cn/law/detailSearchOne?LawID=398754&Query=%E5%88%91%E6%B3%95&IsExact=&PageIndex=2
							var url = "http://" + context.Request.RequestUri.Host + "/" + news.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;
							if (!string.IsNullOrWhiteSpace(url))
							{
								var request = context.CreateNewRequest(url);
								Uri uri = new Uri(url);
								string LawID = uri.Query.Replace("?", "").Split("&")[0].Split("=")[1];
								request.SetProperty("LawID", LawID);
								request.SetProperty("Detailpage", "1");
								request.Name = "detail";
								context.AddFollowRequests(request);
							}
						}
						var pagecount = context.Selectable.Select(Selectors.XPath(".//span[@id='pagecount']"))?.Value;
						if (pagecount != null && Convert.ToInt32(pagecount) > 1)
						{
							for (int i = 2; i <= Convert.ToInt32(pagecount); i++)
							{
								string Listpage = i.ToString();
								string start = context.Request.GetProperty("start");
								string end = context.Request.GetProperty("end");
								Request req = new Request($"http://search.chinalaw.gov.cn/AdvanceSearchResult?SiteID=124&PageIndex={Listpage}&c1={start}&c2={end}&c3=&c4=&title=&Query=");
								req.SetProperty("start", start);
								req.SetProperty("end", end);
								req.Name = "list";

								//req.SetProperty("page", page);
								context.AddFollowRequests(req);
							}
						}
					}
					else
					{
						using (var conn = new SqlConnection(Options_.StorageConnectionString))
						{
							conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(context.Request.RequestUri.AbsoluteUri) + ",  w_list=null:ListNewsParser,Parse' ,'" + DateTime.Now.ToString() + "') ");
						}
					}
					return Task.CompletedTask;
				}
				catch (Exception ex)
				{
					using (var conn = new SqlConnection(Options_.StorageConnectionString))
					{
						conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message+ex.StackTrace) + ",  :ListNewsParser,Parse' ,'" + DateTime.Now.ToString() + "' )");
					}
					return Task.CompletedTask;
				}
			}
		}

		protected class NewsParser : DataParser
		{
			public NewsParser()
			{
				//AddRequiredValidator("search\\.chinalaw\\.gov\\.cn/law/searchTitleDetail");
				AddRequiredValidatorEx("detail");
			}

			protected override Task Parse(DataContext context)
			{
				var typeName = typeof(News).FullName;
				News news = new News();
				news.LawType = "ChinaLaw";
				news.Path = context.Request.RequestUri.AbsoluteUri;
				news.LawID = context.Request.Properties["LawID"]?.Trim();
				if (context.Selectable.Select(Selectors.XPath(".//div[@class='conTit']")) != null)
				{
					news.Title = context.Selectable.Select(Selectors.XPath(".//div[@class='conTit']")).Value?.Replace(" ", "");
					string Detailpage = context.Request.GetProperty("Detailpage");
					news.Detailpage = Detailpage;
					try
					{
						var newsShort = context.Selectable.Select(Selectors.XPath(".//table[@class='d_infor']")).SelectList(Selectors.XPath(".//td")).ToList();
						for (int i = 0; i < newsShort.Count; i += 2)
						{
							var title = newsShort[i]?.Value.Replace(" ", "").Replace("：", "").Replace(":", "");
							var content = newsShort[i + 1]?.Value.Replace(" ", "").Replace("：", "").Replace(":", "");
							if (title == "公布机关" || title == "公布部门")
							{
								news.Organ = content;
							}
							else if (title == "公布日期")
							{
								news.StartDate = content;

							}
							else if (title == "施行日期")
							{
								news.ImplementDate = content;

							}
							else if (title == "效力")
							{
								news.Valid = content;

							}
							else if (title == "门类")
							{
								news.Type = content;
							}
							else
							{
								news.Other += title + "：" + content;
							}
						}
						NewsContent newscontent;
						var contentlist = context.Selectable.Select(Selectors.XPath(".//div[@id='water_div']"));
						if (contentlist != null)
						{
							string a1 = "	";
							string b1 = "	";
							string c1 = "　";

							string content = contentlist.Value.Replace(a1, c1).Replace(b1, c1).Replace(" ", c1);
							MatchCollection mcj = Regex.Matches(content, "&.*?;");
							for (int j = 0; j < mcj.Count; j++)
							{
								content = content.Replace(mcj[j].Groups[0].Value, c1);
							}


							//Regex reg = new Regex("第.*?条 ");
							MatchCollection mc = Regex.Matches(content, "第.{1,6}?条" + c1);

							List<string> strs = new List<string>();
							for (int i = 0; i < mc.Count; i++)
							{
								if (mc[i].Groups[0].Value.Length <= 8)
								{
									strs.Add(mc[i].Groups[0].Value);
								}
							}

							if (strs.Count < 1)
							{
								newscontent = new NewsContent();
								newscontent.Content = content;
								newscontent.LawID = news.LawID;
								newscontent.PathName = news.Title;
								newscontent.Orders = news.LawID + "-" + Detailpage.PadLeft(3, '0') + "-aaaaa";
								news.newscontents.Add(newscontent);
							}
							else
							{

								string[] strArray = strs.ToArray();
								string[] c = content.Split(strArray, StringSplitOptions.RemoveEmptyEntries);

								if (strArray.Length != 0 && strArray.Length + 1 == c.Length)
								{
									for (int i = 0; i < c.Length; i++)
									{

										newscontent = new NewsContent();
										//string ctt = c[i].Replace("\n", "").Replace("\t", "").Replace("\r", "").Replace("；", "。").Replace(";", "。");
										string ctt = c[i].Replace(a1, "").Replace(b1, "").Replace(c1, "").Replace("\n", "").Replace(" ", "").Replace("\t", "").Replace("\r", "").Replace("；", "。").Replace(";", "。");
										string ct = string.Empty;
										string[] c12 = ctt.Split('。', StringSplitOptions.RemoveEmptyEntries);
										for (int h = 0; h < c12.Length; h++)
										{
											if (c12[h].Replace(" ", "").Length > 0)
											{
												ct += "<p>    " + c12[h] + "。</p><br>";
											}
										}
										ct += "<hr>";
										newscontent.Content = i == 0 ? c[i] : "<font>" + strArray[i - 1] + "</font><br>" + "    " + ct;
										newscontent.LawID = news.LawID;
										newscontent.PathName = news.Title + "=>" + (i == 0 ? "" : strArray[i - 1]);
										newscontent.Orders = news.LawID + "-" + Detailpage.PadLeft(3, '0') + "-" + i.ToString().PadLeft(5, '0');
										news.newscontents.Add(newscontent);
									}
								}
							}


						}



						var totalpage = context.Selectable.Select(Selectors.XPath(".//span[@id='pagecount']"))?.Value;
						if (totalpage != null && Detailpage == "1")
						{
							for (int i = 2; i <= Convert.ToInt32(totalpage); i++)
							{
								var url = $"http://search.chinalaw.gov.cn/law/searchTitleDetail?LawID={news.LawID}&Query=&IsExact=&PageIndex={i}";
								if (!string.IsNullOrWhiteSpace(url))
								{
									var request = context.CreateNewRequest(url);
									Uri uri = new Uri(url);
									request.SetProperty("LawID", news.LawID);
									request.SetProperty("Detailpage", i.ToString());
									request.Name = "detail";
									context.AddFollowRequests(request);
								}
							}
						}

						context.AddData(typeName, news);
					}
					catch (Exception ex)
					{
						using (var conn = new SqlConnection(Options_.StorageConnectionString))
						{
							conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message + ex.StackTrace + news.Title+ news.Path) + ",  :NewsParser,Parse' ,'" + DateTime.Now.ToString() + "') ");
						}
						return Task.CompletedTask;
					}
				}
				else
				{
					using (var conn = new SqlConnection(Options_.StorageConnectionString))
					{
						conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(news.Title+ news.Path) + ",  :NewsParser,Parse' ,'" + DateTime.Now.ToString() + "') ");
					}
					return Task.CompletedTask;
				}
				return Task.CompletedTask;
			}
		}

		protected class News
		{
			public string LawID { get; set; }
			public string Title { get; set; }
			public string Organ { get; set; }

			public string StartDate { get; set; }
			public string ImplementDate { get; set; }
			public string Type { get; set; }
			public string Valid { get; set; } = "有效";

			public string Other { get; set; }

			public List<NewsContent> newscontents = new List<NewsContent>();
			public string Detailpage { get; set; }

			public string Path { get; set; }
			public string LawType { get; set; }

		}
		protected class NewsContent
		{
			public NewsContent()
			{
				ContentId = Guid.NewGuid().ToString();
			}
			public string ContentId { get; set; } 
			public string LawID { get; set; }
			public string PathName { get; set; }
			public string Content { get; set; }
			public string Orders { get; set; }

			public int times { get; set; } = 0;//引用次数

		}








	}
}
